pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_branch_prefix.py \
- src/ostbuild/pyostbuild/builtin_build_components.py \
+ src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
- src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
+ src/ostbuild/pyostbuild/builtin_import_tree.py \
src/ostbuild/pyostbuild/builtin_pull_components.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
- src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
- src/ostbuild/pyostbuild/builtin_tree_to_src.py \
src/ostbuild/pyostbuild/builtin_init.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
--- /dev/null
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import hashlib
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class BuildOptions(object):
+ pass
+
+class OstbuildBuild(builtins.Builtin):
+ name = "build"
+ short_description = "Build multiple components and generate trees"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _get_ostbuild_chroot_args(self, architecture):
+ current_machine = os.uname()[4]
+ if current_machine != architecture:
+ args = ['setarch', architecture]
+ else:
+ args = []
+ args.extend(['ostbuild', 'chroot-compile-one',
+ '--snapshot=' + self.snapshot_path])
+ return args
+
+ def _launch_debug_shell(self, architecture, component, cwd=None):
+ args = self._get_ostbuild_chroot_args(architecture)
+ args.extend(['--arch=' + architecture,
+ '--name=' + component,
+ '--debug-shell'])
+ run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
+ fatal("Exiting after debug shell")
+
+ def _build_one_component(self, component, architecture):
+ basename = component['name']
+ branch = component['branch']
+
+ archname = '%s/%s' % (basename, architecture)
+ buildname = 'components/%s' % (archname, )
+
+ current_vcs_version = component.get('revision')
+
+ expanded_component = self.expand_component(component)
+
+ # TODO - deduplicate this with chroot_compile_one
+ current_meta_io = StringIO()
+ json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True)
+ current_metadata_text = current_meta_io.getvalue()
+ sha = hashlib.sha256()
+ sha.update(current_metadata_text)
+ current_meta_digest = sha.hexdigest()
+
+ if (self.buildopts.force_rebuild or
+ basename in self.force_build_components):
+ previous_build_version = None
+ else:
+ previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', buildname],
+ stderr=open('/dev/null', 'w'),
+ none_on_error=True)
+ if (current_vcs_version is not None
+ and previous_build_version is not None):
+ log("Previous build of '%s' is %s" % (archname, previous_build_version))
+
+ previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', previous_build_version,
+ '/_ostbuild-meta.json'],
+ log_initiation=True)
+ sha = hashlib.sha256()
+ sha.update(previous_metadata_text)
+ previous_meta_digest = sha.hexdigest()
+
+ if current_meta_digest == previous_meta_digest:
+ log("Metadata is unchanged from previous")
+ return previous_build_version
+ else:
+ previous_metadata = json.loads(previous_metadata_text)
+ previous_vcs_version = previous_metadata['revision']
+ if current_vcs_version == previous_vcs_version:
+ log("Metadata differs; VCS version unchanged")
+ if self.buildopts.skip_vcs_matches:
+ return previous_build_version
+ for k,v in expanded_component.iteritems():
+ previous_v = previous_metadata.get(k)
+ if v != previous_v:
+ log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
+ else:
+ log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
+ else:
+ log("No previous build for '%s' found" % (archname, ))
+
+ checkoutdir = os.path.join(self.workdir, 'checkouts')
+ component_src = os.path.join(checkoutdir, archname)
+ fileutil.ensure_parent_dir(component_src)
+ run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+ '--checkoutdir=' + component_src,
+ '--clean', '--overwrite', basename])
+
+ artifact_meta = dict(component)
+
+ logdir = os.path.join(self.workdir, 'logs', archname)
+ fileutil.ensure_dir(logdir)
+ log_path = os.path.join(logdir, 'compile.log')
+ if os.path.isfile(log_path):
+ curtime = int(time.time())
+ saved_name = os.path.join(logdir, 'compile-prev.log')
+ os.rename(log_path, saved_name)
+
+ log("Logging to %s" % (log_path, ))
+ f = open(log_path, 'w')
+ chroot_args = self._get_ostbuild_chroot_args(architecture)
+ chroot_args.extend(['--name=' + basename, '--arch=' + architecture])
+ if self.buildopts.shell_on_failure:
+ ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
+ if ecode != 0:
+ self._launch_debug_shell(architecture, basename, cwd=component_src)
+ else:
+ run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
+
+ args = ['ostree', '--repo=' + self.repo,
+ 'commit', '-b', buildname, '-s', 'Build',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged']
+
+ setuid_files = artifact_meta.get('setuid', [])
+ statoverride_path = None
+ if len(setuid_files) > 0:
+ (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
+ f = os.fdopen(fd, 'w')
+ for path in setuid_files:
+ f.write('+2048 ' + path)
+ f.close()
+ args.append('--statoverride=' + statoverride_path)
+
+ component_resultdir = os.path.join(self.workdir, 'results', archname)
+
+ run_sync(args, cwd=component_resultdir)
+ if statoverride_path is not None:
+ os.unlink(statoverride_path)
+
+ return run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', buildname])
+
+ def _compose_one_target(self, target, component_build_revs):
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+
+ compose_contents = [(base_name, base_revision, '/')]
+ for tree_content in target['contents']:
+ name = tree_content['name']
+ rev = component_build_revs[name]
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((name, rev, subpath))
+
+ (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ related_f = os.fdopen(related_fd, 'w')
+ resolved_refs = {}
+ for (name, branch, subpath) in compose_contents:
+ resolved_refs[name] = branch
+ for (name, rev) in resolved_refs.iteritems():
+ related_f.write(name)
+ related_f.write(' ')
+ related_f.write(rev)
+ related_f.write('\n')
+ related_f.close()
+
+ (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ contents_f = os.fdopen(contents_fd, 'w')
+ for (name, branch, subpath) in compose_contents:
+ contents_f.write(branch)
+ contents_f.write('\0')
+ contents_f.write(subpath)
+ contents_f.write('\0')
+ contents_f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--no-triggers', '--union',
+ '--from-file=' + contents_tmppath, compose_rootdir])
+ os.unlink(contents_tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(self.snapshot, f, indent=4, sort_keys=True)
+ f.close()
+
+ treename = 'trees/%s' % (target['name'], )
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'commit', '-b', treename, '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--related-objects-file=' + related_tmppath,
+ '--skip-if-unchanged'], cwd=compose_rootdir)
+ os.unlink(related_tmppath)
+ shutil.rmtree(compose_rootdir)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('--force-rebuild', action='store_true')
+ parser.add_argument('--skip-vcs-matches', action='store_true')
+ parser.add_argument('--no-compose', action='store_true')
+ parser.add_argument('--compose-only', action='store_true')
+ parser.add_argument('--shell-on-failure', action='store_true')
+ parser.add_argument('--debug-shell', action='store_true')
+ parser.add_argument('components', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
+
+ self.buildopts = BuildOptions()
+ self.buildopts.shell_on_failure = args.shell_on_failure
+ self.buildopts.force_rebuild = args.force_rebuild
+ self.buildopts.skip_vcs_matches = args.skip_vcs_matches
+
+ self.force_build_components = set()
+
+ components = self.snapshot['components']
+
+ prefix = self.snapshot['prefix']
+ base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix)
+
+ architectures = self.snapshot['architectures']
+
+ component_to_arches = {}
+
+ runtime_components = []
+ devel_components = []
+
+ for component in components:
+ name = component['name']
+
+ is_runtime = component.get('component', 'runtime') == 'runtime'
+
+ if is_runtime:
+ runtime_components.append(component)
+ devel_components.append(component)
+
+ is_noarch = component.get('noarch', False)
+ if is_noarch:
+ # Just use the first specified architecture
+ component_arches = [architectures[0]]
+ else:
+ component_arches = component.get('architectures', architectures)
+ component_to_arches[name] = component_arches
+
+ for name in args.components:
+ component = components.get(name)
+ if component is None:
+ fatal("Unknown component %r" % (name, ))
+ self.force_build_components.add(name)
+
+ components_to_build = []
+ component_skipped_count = 0
+
+ component_build_revs = {}
+
+ if not args.compose_only:
+ for component in components:
+ for architecture in architectures:
+ components_to_build.append((component, architecture))
+
+ log("%d components to build" % (len(components_to_build), ))
+ for (component, architecture) in components_to_build:
+ archname = '%s/%s' % (component['name'], architecture)
+ build_rev = self._build_one_component(component, architecture)
+ component_build_revs[archname] = build_rev
+
+ targets_list = []
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in architectures:
+ target = {}
+ targets_list.append(target)
+ target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
+
+ base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
+ target['base'] = {'name': base_ref}
+
+ if target_component_type == 'runtime':
+ target_components = runtime_components
+ else:
+ target_components = devel_components
+
+ contents = []
+ for component in target_components:
+ builds_for_component = component_to_arches[component['name']]
+ if architecture not in builds_for_component:
+ continue
+ binary_name = '%s/%s' % (component['name'], architecture)
+ component_ref = {'name': binary_name}
+ if target_component_type == 'runtime':
+ component_ref['trees'] = ['/runtime']
+ else:
+ component_ref['trees'] = ['/runtime', '/devel', '/doc']
+ contents.append(component_ref)
+ target['contents'] = contents
+
+ for target in targets_list:
+ self._compose_one_target(target, component_build_revs)
+
+builtins.register(OstbuildBuild)
+++ /dev/null
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,subprocess,tempfile,re,shutil
-import argparse
-import time
-import urlparse
-import hashlib
-import json
-from StringIO import StringIO
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from .subprocess_helpers import run_sync_monitor_log_file
-from . import ostbuildrc
-from . import buildutil
-from . import fileutil
-from . import kvfile
-from . import odict
-from . import vcs
-
-class BuildOptions(object):
- pass
-
-class OstbuildBuildComponents(builtins.Builtin):
- name = "build-components"
- short_description = "Build multiple components from given source snapshot"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def _get_ostbuild_chroot_args(self, architecture):
- current_machine = os.uname()[4]
- if current_machine != architecture:
- args = ['setarch', architecture]
- else:
- args = []
- args.extend(['ostbuild', 'chroot-compile-one',
- '--snapshot=' + self.snapshot_path])
- return args
-
- def _launch_debug_shell(self, architecture, component, cwd=None):
- args = self._get_ostbuild_chroot_args(architecture)
- args.extend(['--arch=' + architecture,
- '--name=' + component,
- '--debug-shell'])
- run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
- fatal("Exiting after debug shell")
-
- def _build_one_component(self, basename, component, architecture):
- branch = component['branch']
-
- name = '%s/%s' % (basename, architecture)
- buildname = 'components/%s' % (name, )
-
- current_vcs_version = component.get('revision')
-
- # TODO - deduplicate this with chroot_compile_one
- current_meta_io = StringIO()
- meta_copy = dict(component)
- meta_copy['name'] = basename # Note we have to match the name here
- json.dump(meta_copy, current_meta_io, indent=4, sort_keys=True)
- current_metadata_text = current_meta_io.getvalue()
- sha = hashlib.sha256()
- sha.update(current_metadata_text)
- current_meta_digest = sha.hexdigest()
-
- if (self.buildopts.force_rebuild or
- basename in self.force_build_components):
- previous_build_version = None
- else:
- previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', buildname],
- stderr=open('/dev/null', 'w'),
- none_on_error=True)
- if (current_vcs_version is not None
- and previous_build_version is not None):
- log("Previous build of '%s' is %s" % (name, previous_build_version))
-
- previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', previous_build_version,
- '/_ostbuild-meta.json'],
- log_initiation=True)
- sha = hashlib.sha256()
- sha.update(previous_metadata_text)
- previous_meta_digest = sha.hexdigest()
-
- if current_meta_digest == previous_meta_digest:
- log("Metadata is unchanged from previous")
- return False
- else:
- previous_metadata = json.loads(previous_metadata_text)
- previous_vcs_version = previous_metadata['revision']
- if current_vcs_version == previous_vcs_version:
- log("Metadata differs; VCS version unchanged")
- if self.buildopts.skip_vcs_matches:
- return False
- for k,v in meta_copy.iteritems():
- previous_v = previous_metadata.get(k)
- if v != previous_v:
- log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
- else:
- log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
- else:
- log("No previous build for '%s' found" % (name, ))
-
- checkoutdir = os.path.join(self.workdir, 'checkouts')
- fileutil.ensure_dir(checkoutdir)
- component_src = os.path.join(checkoutdir, basename)
- run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
- '--checkoutdir=' + component_src,
- '--clean', '--overwrite', basename])
-
- artifact_meta = dict(component)
-
- logdir = os.path.join(self.workdir, 'logs', name)
- fileutil.ensure_dir(logdir)
- log_path = os.path.join(logdir, 'compile.log')
- if os.path.isfile(log_path):
- curtime = int(time.time())
- saved_name = os.path.join(logdir, 'compile-prev.log')
- os.rename(log_path, saved_name)
-
- log("Logging to %s" % (log_path, ))
- f = open(log_path, 'w')
- chroot_args = self._get_ostbuild_chroot_args(architecture)
- chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
- if self.buildopts.shell_on_failure:
- ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
- if ecode != 0:
- self._launch_debug_shell(architecture, basename, cwd=component_src)
- else:
- run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
-
- args = ['ostree', '--repo=' + self.repo,
- 'commit', '-b', buildname, '-s', 'Build',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--skip-if-unchanged']
-
- setuid_files = artifact_meta.get('setuid', [])
- statoverride_path = None
- if len(setuid_files) > 0:
- (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
- f = os.fdopen(fd, 'w')
- for path in setuid_files:
- f.write('+2048 ' + path)
- f.close()
- args.append('--statoverride=' + statoverride_path)
-
- component_resultdir = os.path.join(self.workdir, 'results', name)
-
- run_sync(args, cwd=component_resultdir)
- if statoverride_path is not None:
- os.unlink(statoverride_path)
- return True
-
- def _resolve_refs(self, refs):
- args = ['ostree', '--repo=' + self.repo, 'rev-parse']
- args.extend(refs)
- output = run_sync_get_output(args)
- return output.split('\n')
-
- def _save_bin_snapshot(self, components, component_architectures):
- bin_snapshot = dict(self.snapshot)
-
- del bin_snapshot['00ostree-src-snapshot-version']
- bin_snapshot['00ostree-bin-snapshot-version'] = 1
-
- for target in bin_snapshot['targets']:
- base = target['base']
- base_name = 'bases/%s' % (base['name'], )
- if 'ostree-revision' not in target:
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_name])
- base['ostree-revision'] = base_revision
-
- if 'architecture-buildroots2' in bin_snapshot:
- for arch,buildroot in bin_snapshot['architecture-buildroots2'].iteritems():
- name = buildroot['name']
- if 'ostree-revision' not in buildroot:
- rev = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', name])
- buildroot['ostree-revision'] = rev
-
- component_revisions = bin_snapshot.get('component-revisions', {})
-
- component_refs_to_resolve = []
- for name in components.iterkeys():
- for architecture in component_architectures[name]:
- archname = '%s/%s' % (name, architecture)
- if archname not in component_revisions:
- component_refs_to_resolve.append('components/' + archname)
-
- if len(component_refs_to_resolve) > 0:
- resolved_refs = self._resolve_refs(component_refs_to_resolve)
- for name,rev in zip(component_refs_to_resolve, resolved_refs):
- assert name.startswith('components/')
- archname = name[len('components/'):]
- component_revisions[archname] = rev
-
- bin_snapshot['component-revisions'] = component_revisions
-
- path = self.get_bin_snapshot_db().store(bin_snapshot)
- log("Binary snapshot: %s" % (path, ))
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--force-rebuild', action='store_true')
- parser.add_argument('--skip-vcs-matches', action='store_true')
- parser.add_argument('--prefix')
- parser.add_argument('--src-snapshot')
- parser.add_argument('--compose', action='store_true')
- parser.add_argument('--compose-only', action='store_true')
- parser.add_argument('--start-at')
- parser.add_argument('--shell-on-failure', action='store_true')
- parser.add_argument('--debug-shell', action='store_true')
- parser.add_argument('components', nargs='*')
-
-
-
- args = parser.parse_args(argv)
- self.args = args
-
- self.parse_config()
- self.parse_snapshot(args.prefix, args.src_snapshot)
-
- component_revisions = self.snapshot.get('component-revisions', {})
-
- if component_revisions is not None:
- snapshot_type = "source+binary"
- else:
- snapshot_type = "source"
-
- log("Using %s snapshot: %s" % (snapshot_type,
- os.path.basename(self.snapshot_path), ))
-
- self.buildopts = BuildOptions()
- self.buildopts.shell_on_failure = args.shell_on_failure
- self.buildopts.force_rebuild = args.force_rebuild
- self.buildopts.skip_vcs_matches = args.skip_vcs_matches
-
- self.force_build_components = set()
-
- required_components = {}
- component_architectures = {}
- for target in self.snapshot['targets']:
- for tree_content in target['contents']:
- (name, arch) = tree_content['name'].rsplit('/', 1)
- required_components[name] = self.snapshot['components'][name]
- if name not in component_architectures:
- component_architectures[name] = set([arch])
- else:
- component_architectures[name].add(arch)
-
- build_component_order = []
- if len(args.components) == 0:
- tsorted = buildutil.tsort_components(required_components, 'build-depends')
- tsorted.reverse()
- build_component_order = tsorted
- else:
- if args.start_at is not None:
- fatal("Can't specify --start-at with component list")
- for name in args.components:
- found = False
- component = self.snapshot['components'].get(name)
- if component is None:
- fatal("Unknown component %r" % (name, ))
- build_component_order.append(name)
- self.force_build_components.add(name)
-
- start_at_index = -1
- if args.start_at is not None:
- for i,component_name in enumerate(build_component_order):
- if component_name == args.start_at:
- start_at_index = i
- break
- if start_at_index == -1:
- fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
- else:
- start_at_index = 0
-
- components_to_build = []
- component_skipped_count = 0
-
- if not args.compose_only:
- for component_name in build_component_order[start_at_index:]:
- component = required_components[component_name]
- architectures = component_architectures[component_name]
- for architecture in architectures:
- archname = '%s/%s' % (component_name, architecture)
- if (component_revisions is not None and
- archname in component_revisions):
- component_skipped_count += 1
- else:
- components_to_build.append((component_name, component, architecture))
-
- log("%d components to build" % (len(components_to_build), ))
- if component_skipped_count > 0:
- log("%d components skipped due to existing component-revisions" % (component_skipped_count, ))
- for (component_name, component, architecture) in components_to_build:
- self._build_one_component(component_name, component, architecture)
-
- self._save_bin_snapshot(required_components, component_architectures)
-
- if args.compose or args.compose_only:
- run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
-
-builtins.register(OstbuildBuildComponents)
component_name = args.component
found = False
- component = self.get_component_meta(component_name)
+ component = self.get_expanded_component(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
is_local = (keytype == 'local')
else:
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
- vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
- component['revision'],
- overwrite=args.overwrite)
+ vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+ component['revision'],
+ overwrite=args.overwrite)
if args.clean:
if is_local:
short_description = "Build artifacts from the current source directory in a chroot"
def _resolve_refs(self, refs):
+ if len(refs) == 0:
+ return []
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
- dependencies = buildutil.build_depends(component_name, components)
- component = components.get(component_name)
+ component = None
+ build_dependencies = []
+ for component in components:
+ if component['name'] == component_name:
+ break
+ build_dependencies.append(component)
ref_to_rev = {}
- arch_buildroot_name = None
- arch_buildroot_rev = None
- if 'architecture-buildroots2' in self.snapshot:
- buildroots = self.snapshot['architecture-buildroots2']
- arch_buildroot = buildroots[architecture]
- arch_buildroot_name = arch_buildroot['name']
- arch_buildroot_rev = arch_buildroot.get('ostree-revision')
- else:
- buildroots = self.snapshot['architecture-buildroots']
- arch_rev_suffix = buildroots[architecture]
- arch_buildroot_name = 'bases/' + arch_rev_suffix
+ arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
+ self.snapshot['prefix'],
+ architecture)
- if arch_buildroot_rev is None:
- arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
- arch_buildroot_name]).strip()
+ arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
+ arch_buildroot_name]).strip()
ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
checkout_trees = [(arch_buildroot_name, '/')]
refs_to_resolve = []
- for dependency_name in dependencies:
- buildname = 'components/%s/%s' % (dependency_name, architecture)
+ for dependency in build_dependencies:
+ buildname = 'components/%s/%s' % (dependency['name'], architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--pristine', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
else:
component_name = self.get_component_from_cwd()
- components = self.snapshot['components']
- component = components.get(component_name)
- if component is None:
- fatal("Couldn't find component '%s' in manifest" % (component_name, ))
- self.metadata = dict(component)
- self.metadata['name'] = component_name
- if not args.pristine:
- self.metadata['src'] = 'dirty:worktree'
- self.metadata['revision'] = 'dirty-worktree'
+ component = self.get_expanded_component(component_name)
workdir = self.workdir
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
- json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
+ json.dump(component, output_metadata, indent=4, sort_keys=True)
output_metadata.close()
chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
recorded_meta_f = open(recorded_meta_path, 'w')
- json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True)
+ json.dump(component, recorded_meta_f, indent=4, sort_keys=True)
recorded_meta_f.close()
builtins.register(OstbuildChrootCompileOne)
+++ /dev/null
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,subprocess,tempfile,re,shutil
-import argparse
-import time
-import urlparse
-import json
-from StringIO import StringIO
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from .subprocess_helpers import run_sync_monitor_log_file
-from . import ostbuildrc
-from . import buildutil
-from . import fileutil
-from . import kvfile
-from . import odict
-from . import vcs
-
-class OstbuildCompose(builtins.Builtin):
- name = "compose"
- short_description = "Build complete trees from components"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def _compose_one_target(self, bin_snapshot, target):
- components = bin_snapshot['component-revisions']
- base = target['base']
- base_name = 'bases/%s' % (base['name'], )
- base_revision = target['base']['ostree-revision']
-
- compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
- if os.path.isdir(compose_rootdir):
- shutil.rmtree(compose_rootdir)
- os.mkdir(compose_rootdir)
-
- compose_contents = [(base_revision, '/')]
- for tree_content in target['contents']:
- name = tree_content['name']
- rev = components[name]
- subtrees = tree_content['trees']
- for subpath in subtrees:
- compose_contents.append((rev, subpath))
-
- (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
- (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
- related_f = os.fdopen(related_fd, 'w')
- contents_f = os.fdopen(contents_fd, 'w')
- for (branch, subpath) in compose_contents:
- related_f.write(' ')
- related_f.write(branch)
- related_f.write('\n')
- contents_f.write(branch)
- contents_f.write('\0')
- contents_f.write(subpath)
- contents_f.write('\0')
- related_f.close()
- contents_f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode', '--no-triggers', '--union',
- '--from-file=' + contents_tmppath, compose_rootdir])
- os.unlink(contents_tmppath)
-
- contents_path = os.path.join(compose_rootdir, 'contents.json')
- f = open(contents_path, 'w')
- json.dump(bin_snapshot, f, indent=4, sort_keys=True)
- f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'commit', '-b', target['name'], '-s', 'Compose',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--related-objects-file=' + related_tmppath,
- '--skip-if-unchanged'], cwd=compose_rootdir)
- os.unlink(related_tmppath)
- shutil.rmtree(compose_rootdir)
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--prefix')
- parser.add_argument('--bin-snapshot')
-
- args = parser.parse_args(argv)
- self.args = args
-
- self.parse_config()
- self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
-
- log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
-
- for target in self.bin_snapshot['targets']:
- log("Composing target %r from %u components" % (target['name'],
- len(target['contents'])))
- self._compose_one_target(self.bin_snapshot, target)
-
-builtins.register(OstbuildCompose)
--- /dev/null
+# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildImportTree(builtins.Builtin):
+ name = "import-tree"
+ short_description = "Extract source data from tree"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def bin_snapshot_to_src(self, bin_snapshot):
+ del bin_snapshot['00ostree-bin-snapshot-version']
+
+ src_snapshot = dict(bin_snapshot)
+ src_snapshot['00ostree-src-snapshot-version'] = 0
+
+ return src_snapshot
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--tree')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+ if args.prefix:
+ self.prefix = args.prefix
+
+ if args.tree:
+ self.load_bin_snapshot_from_path(args.tree)
+ else:
+ self.load_bin_snapshot_from_current()
+
+ snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
+ db = self.get_src_snapshot_db()
+ path = db.store(snapshot)
+ log("Source snapshot: %s" % (path, ))
+
+builtins.register(OstbuildImportTree)
+++ /dev/null
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,stat,subprocess,tempfile,re,shutil
-from StringIO import StringIO
-import json
-import select,time
-import argparse
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-
-class OstbuildModifySnapshot(builtins.Builtin):
- name = "modify-snapshot"
- short_description = "Change the current source snapshot"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--prefix')
- parser.add_argument('--src-snapshot')
-
- args = parser.parse_args(argv)
-
- self.parse_config()
- self.parse_snapshot(args.prefix, args.src_snapshot)
-
- component_name = self.get_component_from_cwd()
- current_meta = self.get_component_meta(component_name)
-
- new_snapshot = dict(self.snapshot)
- new_meta = dict(current_meta)
- if 'patches' in new_meta:
- del new_meta['patches']
- new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
- new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
-
- new_snapshot['components'][component_name] = new_meta
-
- db = self.get_src_snapshot_db()
- path = db.store(new_snapshot)
- log("Replaced %s with %s %s" % (component_name, new_meta['src'],
- new_meta['revision']))
- log("New source snapshot: %s" % (path, ))
-
-builtins.register(OstbuildModifySnapshot)
class OstbuildResolve(builtins.Builtin):
name = "resolve"
- short_description = "Download the source code for a given manifest"
+ short_description = "Expand git revisions in source to exact targets"
def __init__(self):
builtins.Builtin.__init__(self)
self.prefix = self.manifest['prefix']
snapshot = copy.deepcopy(self.manifest)
- component_source_list = map(self._resolve_component_meta, self.manifest['components'])
- del snapshot['components']
+ components = map(self._resolve_component_meta, self.manifest['components'])
+ snapshot['components'] = components
if args.fetch:
if len(args.components) == 0:
fetch_components = args.components
for component_name in fetch_components:
found = False
- for component in component_source_list:
+ for component in components:
if component['name'] == component_name:
found = True
break
fetch_components = []
global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
+ snapshot['patches'] = global_patches_meta
(keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
if args.fetch_patches:
global_patches_meta['revision'] = revision
unique_component_names = set()
- for component in component_source_list:
+ for component in components:
(keytype, uri) = vcs.parse_src_key(component['src'])
name = component['name']
component['branch'])
component['revision'] = revision
- config_opts = list(self.manifest['config-opts'])
- config_opts.extend(component.get('config-opts', []))
- component['config-opts'] = config_opts
-
- patch_files = component.get('patches')
- if patch_files is not None:
- component['patches'] = copy.deepcopy(global_patches_meta)
- component['patches']['files'] = patch_files
-
- manifest_architectures = snapshot['architectures']
-
- ostree_prefix = snapshot['prefix']
- base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
-
- snapshot['architecture-buildroots'] = {}
- for architecture in manifest_architectures:
- snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
- # Lame bit neeeded because I didn't have enough foresight to use an object
- # for this in the first place, and I don't want to break backwards compatibility
- # right now.
- snapshot['architecture-buildroots2'] = {}
- for architecture in manifest_architectures:
- snapshot['architecture-buildroots2'][architecture] = {'name': 'bases/%s-%s-devel' % (base_prefix, architecture)}
-
- components_by_name = {}
- component_ordering = []
- build_prev_component = None
- runtime_prev_component = None
- runtime_components = []
- devel_components = []
-
- builds = {}
-
- for component in component_source_list:
- base_name = component['name']
- name = '%s/%s' % (ostree_prefix, base_name)
- component['name'] = name
-
- components_by_name[name] = component
-
- if build_prev_component is not None:
- component['build-depends'] = [build_prev_component['name']]
- build_prev_component = component
-
- is_runtime = component.get('component', 'runtime') == 'runtime'
-
- if runtime_prev_component is not None:
- component['runtime-depends'] = [runtime_prev_component['name']]
-
- if is_runtime:
- runtime_prev_component = component
- runtime_components.append(component)
- devel_components.append(component)
-
- is_noarch = component.get('noarch', False)
- if is_noarch:
- # Just use the first specified architecture
- component_arches = [manifest_architectures[0]]
- else:
- component_arches = component.get('architectures', manifest_architectures)
- builds[name] = component_arches
-
- # We expanded these keys
- del snapshot['config-opts']
- del snapshot['vcsconfig']
- del snapshot['patches']
- del snapshot['architectures']
-
- targets_list = []
- snapshot['targets'] = targets_list
- for target_component_type in ['runtime', 'devel']:
- for architecture in manifest_architectures:
- target = {}
- targets_list.append(target)
- target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
-
- base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
- target['base'] = {'name': base_ref}
-
- if target_component_type == 'runtime':
- target_components = runtime_components
- else:
- target_components = devel_components
-
- contents = []
- for component in target_components:
- builds_for_component = builds[component['name']]
- if architecture not in builds_for_component:
- continue
- binary_name = '%s/%s' % (component['name'], architecture)
- component_ref = {'name': binary_name}
- if target_component_type == 'runtime':
- component_ref['trees'] = ['/runtime']
- else:
- component_ref['trees'] = ['/runtime', '/devel', '/doc']
- contents.append(component_ref)
- target['contents'] = contents
-
- for component in components_by_name.itervalues():
- del component['name']
- snapshot['components'] = components_by_name
-
- snapshot['00ostree-src-snapshot-version'] = 0
-
- current_time = time.time()
-
src_db = self.get_src_snapshot_db()
path = src_db.store(snapshot)
log("Source snapshot: %s" % (path, ))
+++ /dev/null
-# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
-# http://people.gnome.org/~walters/docs/build-api.txt
-
-import os,sys,stat,subprocess,tempfile,re,shutil
-import argparse
-from StringIO import StringIO
-import json
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from . import buildutil
-
-class OstbuildTreeToSrc(builtins.Builtin):
- name = "tree-to-src"
- short_description = "Turn a tree into a source snapshot"
-
- def __init__(self):
- builtins.Builtin.__init__(self)
-
- def bin_snapshot_to_src(self, bin_snapshot):
- del bin_snapshot['00ostree-bin-snapshot-version']
-
- src_snapshot = dict(bin_snapshot)
- src_snapshot['00ostree-src-snapshot-version'] = 0
-
- return src_snapshot
-
- def execute(self, argv):
- parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--prefix')
- parser.add_argument('--tree')
-
- args = parser.parse_args(argv)
- self.parse_config()
- if args.prefix:
- self.prefix = args.prefix
-
- if args.tree:
- self.load_bin_snapshot_from_path(args.tree)
- else:
- self.load_bin_snapshot_from_current()
-
- snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
- db = self.get_src_snapshot_db()
- path = db.store(snapshot)
- log("Source snapshot: %s" % (path, ))
-
-builtins.register(OstbuildTreeToSrc)
'/_ostbuild-meta.json'])
return json.loads(text)
- def get_component_meta(self, name):
- assert self.repo is not None
+ def expand_component(self, component):
+ meta = dict(component)
+ global_patchmeta = self.snapshot.get('patches')
+ if global_patchmeta is not None:
+ component_patch_files = component.get('patches', [])
+ if len(component_patch_files) > 0:
+ patches = dict(global_patchmeta)
+ patches['files'] = component_patch_files
+ meta['patches'] = patches
+ config_opts = self.snapshot.get('config-opts', [])
+ config_opts.extend(component.get('config-opts', []))
+ meta['config-opts'] = config_opts
+ return meta
- if self.snapshot is not None:
- return self.snapshot['components'][name]
+ def get_component(self, name):
+ assert self.repo is not None
+ assert self.snapshot is not None
+ for component in self.snapshot['components']:
+ if component['name'] == name:
+ return component
+ fatal("Couldn't find component '%s' in manifest" % (component_name, ))
- meta = self._meta_cache.get(name)
- if meta is None:
- content = self.get_component_snapshot(name)
- meta = self.get_component_meta_from_revision(content['ostree-revision'])
- self._meta_cache[name] = meta
- return meta
+ def get_expanded_component(self, name):
+ return self.expand_component(self.get_component(name))
def get_prefix(self):
if self.prefix is None:
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
- src_ver = self.snapshot['00ostree-src-snapshot-version']
+ key = '00ostbuild-manifest-version'
+ src_ver = self.snapshot[key]
if src_ver != 0:
- fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0" % (src_ver, ))
+ fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
def parse_bin_snapshot(self, prefix, path):
self.parse_prefix(prefix)
import argparse
from . import builtins
-from . import builtin_build_components
+from . import builtin_build
from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
-from . import builtin_compose
from . import builtin_compile_one
from . import builtin_deploy_root
from . import builtin_deploy_qemu
+from . import builtin_import_tree
from . import builtin_git_mirror
from . import builtin_pull_components
from . import builtin_privhelper_deploy_qemu
from . import builtin_prefix
from . import builtin_resolve
from . import builtin_modify_snapshot
-from . import builtin_tree_to_src
from . import builtin_init
from . import builtin_status
projects / ostree.git / commitdiff